"""
数据规则定义：
目标：
1）增加新品种时，直接按格式增加既可，不用全线路整理；重要的就是协议统一，导入可用
2）支持业务上的使用：比如存储更新，回测遍历；支持多票与单票的处理
原则
1）考虑python回测性能，使用pkl,按文件保存；是mongodb的十倍以上
2）考虑到业务易容性，增加多票的保存模式
协议：
1）pkl文件，亲和python
2）文件结构，
2.1）一层目录，必须：按数据种类：price,factor,event,news
2.2）二层目标，必须：按投资品种存放：stock,future,fund,index,bond,stock_A   _region   同交易日历中的region
2.3）三层目录，必须：按数据类型datatype存放：k1d,knm,k1m,tick,k1m_source   支持多源数据配置与融合
price:
2.4）四层目录，非必须：按复权方式存放；bfq,hfq,qfq  没有这层则意味着不复权，如果有复权因子的也放在这层
2.5）第五层：必须，按合约存放
2.6）数据文件：必须，按时间频率生成单独文件
factor
2.4）四层目录  按因子名称存放
2.5）数据文件：必须，按时间频率生成单独文件
event
2.2）二层目标事件目录
news
3) 多因子，多券，增加独立数据存储；如果没有这个类存储数据则，自动合并成
3.1）独立存储时，第四层目录增加  _all
3.2)  独立存储时，数据文件:按时间频率生成单独文件
4）数据文件生成原则
4.1）频率文件界定：主要用2种，按月存储，按日存储  freq="%Y%m"  or "%Y%m%d" ;tick与k1m下精细度的按日存储；其它的都按月存储
obj_dstr.pkl    all的使用all_sdtr.pkl;数据标准是  日内数据列表存放；按日回测使用
4.2）单合约  按日存储  list列表模式，必须正确的时间序列
4.3）单合约  按月存储  dict模式，values为当日的lists数据，必须正确的时间序列 {date:[]}
4.4）_all 按日存储  dict模式；[]
4.5) _all 按月存储 dict模式：{date:[]}
5) 其它：
5.1）尽可能统一的调用方法
5.2）在研究中尽可能使用简单的头寸结算方式；只依赖行情
5.3）dataset不考虑跨品种的数据工作，价差业务不包含在这里
6.面向回测封装的回测数据
6.1.集成多源合并
6.2.factor合并   单独按因子名字保存
6.3.event合并 简单做下事件时间插入操作
6.4.考虑回测集与测试集的划分
按日，按月 读取
有复权，计算复权
all,还是个股，还是多个个股 读取方式

tick,k1m,k1d:
fund,index,stock:
    1) k1d:all/obj  默认hfq,bfq 复权
    2) k1m:obj
    3) tick:obj
future:
    1) k1m:obj
    2) tick:obj
    3) k1d:obj/all
分步处理
url 上层
是否应该有
"""
from jili.core.printlog import print
import datetime
import os
import multiprocessing
from jili.tool.convert import str2datetime
from jili.core import load,save,load_async,save_async
from jili.data.db import get_calender
import pandas as pd
from jili.data.db import getdb_client,tradecal
from jili.tool.state import config
from research.factor import factordata
from research.calcor.calcors_graph import graph_calcor
from copy import deepcopy
from jili.tool import sortdict
from multiprocessing import Manager
class dataset:
    """
    data综合工具：
    1.数据读取迭代
    2.数据更新保存
    3.各种类型数据都能处理
    4.各种品种数据都能处理
    5.离散小工具：数据最大日期，
    """
    def __init__(self,product="future",datatype="k1d",baseurl=r"F:",region="C",fqtype="hfq",freq=None,source=None,runmode="pkl",iszip=False,isasync=[False,False]):
        """
        Args:
            product: stock,future,fund,index,bond,coin
            datatype: k1d,k1m,tick
            baseurl: price的根目录   r"F:/price"
            region: C,A,H   非C的 会自动拼接成  stock_A
            fqtype: hfq,bfq  跟回测中的数据对应，如果不存在合适的数据则需要计算复权数据
            source:数据源，专用拼接
            runmode：pkl文件，shm:共享内存，未来支持：redis,等内存数据库
        """
        if isasync[0]:
            self.__iml_load = load_async
        else:
            self.__iml_load = load
        if isasync[1]:
            self.__iml_save=save_async
        else:
            self.__iml_save = save
        self.baseurl=baseurl
        self.region=region
        self.product=product
        self.datatype=datatype
        self.fqtype=fqtype
        self.data_tmp={}#数据缓存
        self.factors={}
        self.db=None
        self.iszip=iszip
        self.source=source
        if freq:
            self.freq=freq
        else:
            if datatype in ["k1m","tick","k3m","k2m"]:
                self.freq = "%Y%m%d"
            else:
                self.freq = "%Y%m"
        self.url=baseurl
        if region=="C":
            self.url=os.path.join(self.url,self.product)
        else:
            self.url = os.path.join(self.url, self.product+"_"+self.region)
        self.url = os.path.join(self.url, self.datatype)
        if source:
            self.url = self.url+"_"+source
        if self.product  not in ["index"]:
            if self.fqtype is not None:
                self.url = os.path.join(self.url, self.fqtype)
        self.isavailable=False
        if os.path.exists(self.url):
            l=os.listdir(self.url)
            if len(l)>0:
                self.isavailable=True
        self.runmode=runmode
        self.k1dtool=None
    def get_k1d_bydate(self,obj,d):
        if self.k1dtool is None:
            self.k1dtool=dataset(product=self.product,datatype="k1d",baseurl=self.baseurl,region=self.region,fqtype=self.fqtype,source=self.source)
        data=self.k1dtool.get_data_bydate(obj,d)
        return data
    def get_objs(self):
        objs=os.listdir(self.url)
        return objs
    #基础工具  获取最新日期，最新timekey,最新行情
    def get_maxdate(self,obj):#obj  单个合约或是  all
        maxdate=None
        url = self.url
        if obj=="all":
            url0 = os.path.join(url, "_all")
            if os.path.exists(url0):
                l = max(os.listdir(url0))
                if self.freq=="%Y%m%d":
                    maxdate = l.replace(".pkl","").split("_")[-1]
                    maxdate = str2datetime(maxdate)
                else:
                    d = self.__iml_load(os.path.join(url0, l),iszip=self.iszip)
                    maxdate = max(list(d.keys()))
        else:
            if self.freq == "%Y%m%d":
                url0 = os.path.join(url, obj)
                if os.path.exists(url0):
                    l = max(os.listdir(url0))
                    maxdate = l.replace(".pkl","").split("_")[-1]
                    maxdate=str2datetime(maxdate)
            else:
                url0 = os.path.join(url, obj)
                if os.path.exists(url0):
                    l = max(os.listdir(url0))
                    d = self.__iml_load(os.path.join(url0, l),iszip=self.iszip)
                    maxdate = max(list(d.keys()))
        return maxdate
    def covert_objs2all(self,objs,start=None,end=None):
        datas={}
        for dtype,t  in self.iter_price(objs,start,end):
            tk=t["timekey"]
            obj=t["obj"]
            if tk not in datas.keys():
                datas[tk]={}
            datas[tk][obj]=t
        self.save_byall(datas)
    def get_startdate(self,obj):#obj  单个合约或是  all
        maxdate=None
        url = self.url
        if obj=="all":
            url0 = os.path.join(url, "_all")
            if os.path.exists(url0):
                l = min(os.listdir(url0))
                if self.freq=="%Y%m%d":
                    maxdate = l.replace(".pkl","").split("_")[-1]
                    maxdate = str2datetime(maxdate)
                else:
                    d = self.__iml_load(os.path.join(url0, l),iszip=self.iszip)
                    maxdate = min(list(d.keys()))
        else:
            if self.freq == "%Y%m%d":
                url0 = os.path.join(url, obj)
                if os.path.exists(url0):
                    l = min(os.listdir(url0))
                    maxdate = l.replace(".pkl","").split("_")[-1]
                    maxdate = str2datetime(maxdate)
            else:
                url0 = os.path.join(url, obj)
                if os.path.exists(url0):
                    l = min(os.listdir(url0))
                    d = self.__iml_load(os.path.join(url0, l),iszip=self.iszip)
                    maxdate = min(list(d.keys()))
        return maxdate
    def get_datelist(self,obj):
        datelist = []
        url = self.url
        if obj == "all":
            url0 = os.path.join(url, "_all")
        else:
            url0 = os.path.join(url, obj)
        if os.path.exists(url0):
            l = os.listdir(url0)
            l.sort()
            if self.freq == "%Y%m%d":
                for i in l:
                    d = i.replace(".pkl","").split("_")[-1]
                    datelist.append(d)
            else:
                d0 = self.__iml_load(os.path.join(url0, l[0]),iszip=self.iszip)
                d0 = min(list(d0.keys()))
                d1 = self.__iml_load(os.path.join(url0, l[-1]),iszip=self.iszip)
                d1 = min(list(d1.keys()))
                for d in get_calender(d0, d1):
                    datelist.append(d)
        return datelist

    def get_maxtimekey(self,obj):
        maxtimekey = None
        url = self.url
        if obj == "all":
            url0 = os.path.join(url, "_all")
            if os.path.exists(url0):
                l = max(os.listdir(url0))
                d = self.__iml_load(os.path.join(url0, l),iszip=self.iszip)
                maxdate = max(list(d.keys()))
                maxtimekey=d[maxdate][-1]["timekey"]
        else:
            if self.freq == "%Y%m%d":
                url0 = os.path.join(url, obj)
                if os.path.exists(url0):
                    l = max(os.listdir(url0))
                    d = self.__iml_load(os.path.join(url0, l),iszip=self.iszip)
                    maxtimekey = d[-1]["timekey"]
            else:
                url0 = os.path.join(url, obj)
                if os.path.exists(url0):
                    l = max(os.listdir(url0))
                    d = self.__iml_load(os.path.join(url0, l),iszip=self.iszip)
                    maxdate = max(list(d.keys()))
                    if self.datatype == "k1d":
                        maxtimekey = maxdate
                    else:
                        dd = d[maxdate]
                        maxtimekey = dd[-1]["timekey"]
        return maxtimekey
    def get_lastdata(self,obj):
        data = None
        url = self.url
        if obj == "all":
            url0 = os.path.join(url, "_all")
            if os.path.exists(url0):
                l = max(os.listdir(url0))
                d = self.__iml_load(os.path.join(url0, l),iszip=self.iszip)
                maxdate = max(list(d.keys()))
                data=d[maxdate]
        else:
            if self.freq == "%Y%m%d":
                url0 = os.path.join(url, obj)
                if os.path.exists(url0):
                    l = max(os.listdir(url0))
                    d = self.__iml_load(os.path.join(url0, l),iszip=self.iszip)
                    data = d[-1]
            else:
                url0 = os.path.join(url, obj)
                if os.path.exists(url0):
                    l = max(os.listdir(url0))
                    d = self.__iml_load(os.path.join(url0, l),iszip=self.iszip)
                    maxdate = max(list(d.keys()))
                    if self.datatype == "k1d":
                        data = d[maxdate]
                    else:
                        dd = d[maxdate]
                        data = dd[-1]
        return data
    #获取数，按日期，按开始结束日期需要增加复权数据计算
    def get_factor(self,obj,timekey):
        timekey=pd.to_datetime(timekey)
        if obj  in self.factors.keys():
            if timekey in self.factors[obj].keys():
                return self.factors[obj][timekey]
        if self.db is None:
            self.db=getdb_client(dbname="stock",ip=config.mongodb_ip)
        t=1
        for i in self.db["stock_right_factor"].find({"obj":obj,"timekey":{"$lte":timekey}}).sort([("timekey",-1)]):
            if obj not in self.factors.keys():
                self.factors[obj]={}
            # t=i["adjustFactor"]
            t=i
            self.factors[obj][timekey]=t
            break
        return t
    def get_data_bydate(self,obj,d):#具有快速缓存
        data = []
        d0=str2datetime(d)
        url = self.url
        if isinstance(obj,str):
            if obj in self.data_tmp.keys():
                if d0 in self.data_tmp[obj].keys():
                    return self.data_tmp[obj][d0]
            if obj == "all":
                url0 = os.path.join(url, "_all", "all_" + d0.strftime(self.freq) + ".pkl")
            else:
                url0 = os.path.join(url, obj, obj + "_" + d0.strftime(self.freq) + ".pkl")
            if os.path.exists(url0):
                self.data_tmp={}
                data0 = self.__iml_load(url0, iszip=self.iszip)
                if self.freq == "%Y%m%d":
                    if obj not in self.data_tmp.keys():
                        self.data_tmp[obj]={}
                    self.data_tmp[obj][d0]=data0
                else:
                    self.data_tmp[obj] = data0
                if obj in self.data_tmp.keys():
                    if d0 in self.data_tmp[obj].keys():
                        return self.data_tmp[obj][d0]
        else:
            if d0 in self.data_tmp.keys():
                data0=self.data_tmp[d]
                for obj0 in obj:
                    if obj0 in data0.keys():
                        v=data0[obj0]
                        if isinstance(v,list):
                            data=data+v
                        else:
                            data.append(v)
                    # else:#self.__iml_load
                    #     url0 = os.path.join(url, obj0, obj0 + "_" + d0.strftime(self.freq) + ".pkl")
                    #     if os.path.exists(url0):
                    #         if self.freq == "%Y%m%d":
                    #             v = self.__iml_load(url0,iszip=self.iszip)
                    #             data = data + v
                    #         else:
                    #             data0 = self.__iml_load(url0,iszip=self.iszip)
                    #             for d1,v in data0.items():
                    #                 if d1 not in self.data_tmp.keys():
                    #                     self.data_tmp[d1]={}
                    #                 self.data_tmp[d1][obj0] = v[0]
                    #                 if d1==d0:
                    #                     data.append(v[0])
            else:#self.__iml_load
                self.data_tmp = {}
                for obj0 in obj:
                    url0 = os.path.join(url, obj0, obj0 + "_" + d0.strftime(self.freq) + ".pkl")
                    if os.path.exists(url0):
                        data0 = self.__iml_load(url0, iszip=self.iszip)
                        if self.freq == "%Y%m%d":
                            if d0 not in self.data_tmp.keys():
                                self.data_tmp[d0] = {}
                            self.data_tmp[d0][obj] = data0
                            data = data + data0
                        else:
                            for d1, v in data0.items():
                                if d1 not in self.data_tmp.keys():
                                    self.data_tmp[d1] = {}
                                self.data_tmp[d1][obj0] = v[0]
                                if d1 == d0:
                                    data.append(v[0])
        if data is None:
            data=[]
        return data
    def get_data(self,obj,start=None,end=None):
        """
        Args:
            obj:
            start:date
            end:date
        Returns:
            按日的行情集  如果是个股就是list,如果是all，就是按timekey的dict
        """
        url = self.url
        if obj=="all":
            url0 = os.path.join(url, "_all")
        else:
            url0 = os.path.join(url, obj)
        data = []
        if os.path.exists(url0):
            l=os.listdir(url0)
            l.sort()
            if start:
                d0=str2datetime(start)
                start0=obj+"_"+d0.strftime(self.freq)+".pkl"
            else:
                d0=None
                start0=None
            if end:
                d1=str2datetime(end)
                end0=obj+"_"+d1.strftime(self.freq)+".pkl"
            else:
                d1=None
                end0=None
            if self.freq == "%Y%m%d":
                for i in l:
                    if start:
                        if i<start0:
                            continue
                    if end0:
                        if i>end0:
                            continue
                    url1 = os.path.join(url0, i)
                    dset = self.__iml_load(url1,iszip=self.iszip)
                    data = data + dset
            else:
                isbreak=False
                for i in l:
                    if i<start0:
                        continue
                    # if i==start0:
                    url1 = os.path.join(url0, i)
                    dset = self.__iml_load(url1,iszip=self.iszip)
                    for d, v in dset.items():
                        if d0 is not None:
                            if d < d0:
                                continue
                        if d1 is not None:
                            if d > d1:
                                isbreak=True
                                break
                            data = data + v
                    if isbreak:
                        continue
        else:
            if not isinstance(obj,str):
                for d in get_calender(start,end):
                    for obj0 in obj:
                        data=data+self.get_data_bydate(obj0,d)
        return data
    def objsdata2datas(self,l):
        datas={}
        for i in l:
            obj=i["obj"]
            timekey=i["timekey"]
            if obj not in datas.keys():
                datas[obj]={}
            datas[obj][timekey]=i
        return datas
    #保存数据 普通模式与 ——all模式
    def save_byday(self,obj,datas,isupdate=True,add_url=None):
        if not datas:
            print("save_byday datas数据为空")
            return None
        if self.freq == "%Y%m%d":
            for kk,v in datas.items():
                k=kk.strftime(self.freq)
                url0 = os.path.join(self.url, obj, obj + "_" + k + ".pkl")
                vv=[]
                keys = []
                if isinstance(v,dict):
                    i = v
                    if isupdate:
                        keys.append(i["timekey"])
                    vv.append(i)
                else:
                    for i in v:
                        if isupdate:
                            keys.append(i["timekey"])
                        vv.append(i)
                if os.path.exists(url0) and isupdate:
                    data = self.__iml_load(url0,iszip=self.iszip)
                    for i in reversed(data):
                        if i["timekey"] not in keys:
                            vv.insert(0, i)
                self.__iml_save(vv,url0,iszip=self.iszip)
                if add_url is not None:
                    self.__iml_save(vv, url0.replace(self.baseurl, add_url), iszip=self.iszip)
        else:
            print("不是按日的freq","freq == %Y%m%d")
    def save(self,obj,datas,isupdate=True,add_url=None):
        if isinstance(datas,list):
            datas0={}
            for k in datas:
                d=k["timekey"]
                if self.freq=="%Y%m%d":
                    dstr=d.strftime(self.freq)
                    if dstr not in datas0.keys():
                        datas0[dstr]=[]
                    datas0[dstr].append(k)
                else:
                    dstr = d.strftime(self.freq)
                    if dstr not in datas0.keys():
                        datas0[dstr] = {}
                    day=datetime.datetime(d.year,d.month,d.day)
                    if day not in datas0[dstr].keys():
                        datas0[dstr][day]=[]
                    datas0[dstr][day].append(k)
            datas=datas0
        if not isinstance(datas,dict):
            print("err","save datas必须是按日数据的dict")
            return None
        for k in datas.keys():
            break
        if isinstance(k,str):
            for k , v in datas.items():
                url0=os.path.join(self.url,obj,obj+"_"+k+".pkl")
                if os.path.exists(url0) and isupdate:
                    data = self.__iml_load(url0,iszip=self.iszip)
                    data.update(v)
                    data=sortdict(data,True,"key")
                    self.__iml_save(data,url0,iszip=self.iszip)
                    if add_url is not None:
                        self.__iml_save(data, url0.replace(self.baseurl,add_url),iszip=self.iszip)
                else:
                    self.__iml_save(v,url0,iszip=self.iszip)
                    if add_url is not None:
                        self.__iml_save(v, url0.replace(self.baseurl,add_url),iszip=self.iszip)
        else:
            if self.freq == "%Y%m%d":
                self.save_byday(obj,datas,isupdate,add_url)
            elif obj=="all":
                self.save_byall(datas, isupdate,add_url)
            else:
                self.save_byobj(obj, datas, isupdate,add_url)
    # def listdata2datas(self,l):
    #     rst={}
    #     for i in
    def save_byall(self,datas,isupdate=True,add_url=None):
        if not datas:
            print("save_byall datas数据为空")
            return None
        l=list(datas.keys())
        prek=None
        data={}
        l.sort()
        if isinstance(l[0],str):
            for k , v in datas.items():
                url0=os.path.join(self.url,"_all","all"+"_"+k+".pkl")
                if os.path.exists(url0) and isupdate:
                    data = self.__iml_load(url0,iszip=self.iszip)
                    data.update(v)
                    data=sortdict(data,True,"key")
                    self.__iml_save(data,url0,iszip=self.iszip)
                    if add_url is not None:
                        self.__iml_save(data, url0.replace(self.baseurl,add_url),iszip=self.iszip)
                else:
                    self.__iml_save(v,url0,iszip=self.iszip)
                    if add_url is not None:
                        self.__iml_save(v, url0.replace(self.baseurl,add_url),iszip=self.iszip)
        else:
            for d in l:
                v=datas[d]
                k = d.strftime(self.freq)
                if prek!=k:
                    if data:
                        url0 = os.path.join(self.url, "_all", "all_" + prek + ".pkl")
                        if os.path.exists(url0) and isupdate:
                            data0 = self.__iml_load(url0,iszip=self.iszip)
                            data0.update(data)
                            data0 = sortdict(data0, True, "key")
                            self.__iml_save(data0, url0,iszip=self.iszip)
                            if add_url is not None:
                                self.__iml_save(data0, url0.replace(self.baseurl, add_url), iszip=self.iszip)
                        else:
                            self.__iml_save(data,url0,iszip=self.iszip)
                            if add_url is not None:
                                self.__iml_save(data, url0.replace(self.baseurl, add_url), iszip=self.iszip)
                        if isinstance(v,dict):
                            data = {d: [v]}
                        else:
                            data={d:v}
                    else:
                        if isinstance(v, dict):
                            data[d] = [v]
                        else:
                            data[d] = v
                else:
                    if isinstance(v, dict):
                        data[d]=[v]
                    else:
                        data[d]=v
                prek=k
            if data:
                url0 = os.path.join(self.url, "_all", "all_" + k + ".pkl")
                if os.path.exists(url0) and isupdate:
                    data0 = self.__iml_load(url0,iszip=self.iszip)
                    data0.update(data)
                    data0 = sortdict(data0, True, "key")
                    self.__iml_save(data0, url0,iszip=self.iszip)
                    if add_url is not None:
                        self.__iml_save(data0, url0.replace(self.baseurl,add_url),iszip=self.iszip)
                else:
                    self.__iml_save(data, url0,iszip=self.iszip)
                    if add_url is not None:
                        self.__iml_save(data, url0.replace(self.baseurl,add_url),iszip=self.iszip)
    def save_byobj(self,obj,datas,isupdate=True,add_url=None):
        if not datas:
            print("save_byobj datas数据为空")
            return None
        l=list(datas.keys())
        prek=None
        data={}
        l.sort()
        for d in l:
            v=datas[d]
            k = d.strftime(self.freq)
            if prek!=k:
                if data:
                    url0 = os.path.join(self.url, obj, obj+"_" + prek + ".pkl")
                    if os.path.exists(url0) and isupdate:
                        data0 = self.__iml_load(url0,iszip=self.iszip)
                        data0.update(data)
                        data0 = sortdict(data0, True, "key")
                        self.__iml_save(data0, url0,iszip=self.iszip)
                        if add_url is not None:
                            self.__iml_save(data0, url0.replace(self.baseurl, add_url), iszip=self.iszip)
                    else:
                        self.__iml_save(data,url0,iszip=self.iszip)
                        if add_url is not None:
                            self.__iml_save(data, url0.replace(self.baseurl, add_url), iszip=self.iszip)
                    if isinstance(v,dict):
                        data = {d: [v]}
                    else:
                        data={d:v}
                else:
                    if isinstance(v, dict):
                        data[d] = [v]
                    else:
                        data[d] = v
            else:
                if isinstance(v, dict):
                    data[d]=[v]
                else:
                    data[d]=v
            prek=k
        if data:
            url0 = os.path.join(self.url, obj, obj+"_" + k + ".pkl")
            if os.path.exists(url0) and isupdate:
                data0 = self.__iml_load(url0,iszip=self.iszip)
                data0.update(data)
                data0 = sortdict(data0, True, "key")
                self.__iml_save(data0, url0,iszip=self.iszip)
                if add_url is not None:
                    self.__iml_save(data0, url0.replace(self.baseurl, add_url), iszip=self.iszip)
            else:
                self.__iml_save(data, url0,iszip=self.iszip)
                if add_url is not None:
                    self.__iml_save(data, url0.replace(self.baseurl, add_url), iszip=self.iszip)
    #迭代器，主要是回测用，需要增加复权数据计算
    def get_pkl_byday(self,url0,obj,start,end):
        for d in get_calender(start, end, self.region):
            url1 = os.path.join(url0, obj + "_" + d.strftime(self.freq) + ".pkl")
            datset = self.__iml_load(url1,iszip=self.iszip)
            if datset:
                for i in datset:
                    yield self.datatype, i
            else:
                print("sys", "不存在", obj, d, url1)
    def iter_price_bycalcors_graph_byday(self,obj,start,end,factors,cc,prebatch,factor_url,future_factors={}):
        data=[]
        pred=None
        for dtype,t in self.iter_price_bycalcors_graph(obj,start,end,factors,cc,prebatch,factor_url,future_factors):
            tk=t["timekey"]
            d=tk.date()
            if pred is None:
                pred=d
            if d!=pred:
                pretk=datetime.datetime(pred.year,pred.month,pred.day)
                yield dtype,pretk,data
                data=[]
                pred=d
            data.append(t)
        pretk = datetime.datetime(pred.year, pred.month, pred.day)
        yield dtype, pretk, data
    def iter_price_bycalcors_graph(self,obj,start,end,factors,cc,prebatch,factor_url,future_factors={}):
        #增加计算图的行情迭代
        if start:
            start=tradecal.gettradeday(start)
        if end:
            end=tradecal.gettradeday(end)
        factors_f={}
        calcors_f={}
        for k in factors:
            factors_f[k]=factordata(name=k, url=factor_url, freq=self.freq)
        start0=tradecal.get_diff_tradeday(start,-prebatch,region=self.region)
        if end:
            end = str2datetime(end)
        else:
            now = datetime.datetime.now()
            end = tradecal.getpretradeday(now)
            if now.hour >= 18:
                if tradecal.istradeday(now):
                    end = tradecal.gettradeday(now)
            if future_factors:
                n0 = max(list(future_factors.values()))
                maxds = []
                for k, f in factors_f.items():
                    if k in future_factors.keys():
                        maxd = f.get_maxtimekey()
                        maxds.append(maxd)
                end = min(maxds)
                end = tradecal.get_diff_tradeday(end, -n0)
        for dtype,t in self.iter_price(obj,start0,end):
            obj=t["obj"]
            tk=t["timekey"]
            for k, f in factors_f.items():
                if k in future_factors.keys():
                    ndays = future_factors[k]
                    tk0 = tradecal.get_diff_tradeday(tk, ndays)
                    ff = f.get_bydate(tk0)
                else:
                    ff = f.get_bydate(tk)

                isnohave = True
                if k[:3] == "is_":
                    if obj in ff:
                        t[k] = True
                        isnohave = False
                    else:
                        t[k] = False
                        isnohave = False
                else:
                    if isinstance(ff,dict):
                        if obj in ff.keys():
                            dd0 = ff[obj]
                            if isinstance(dd0, dict):
                                t.update(dd0)
                            else:
                                t[k] = dd0
                            isnohave = False
                        if isnohave:
                            if "ts_code" in t.keys():
                                code = t["ts_code"]
                                if code in ff.keys():
                                    dd0 = ff[code]
                                    if isinstance(dd0, dict):
                                        t.update(dd0)
                                    else:
                                        t[k] = dd0
                                    isnohave = False
                    else:
                        t[k]=ff
                        isnohave = False
                if isnohave:
                    t[k] = None
            if obj not in calcors_f.keys():
                calcors_f[obj] = graph_calcor(deepcopy(cc))
            t = calcors_f[obj].onbar(t)
            if tk>=start:
                yield dtype,t
    def get_price_bycalcors_graph(self,obj,start,end,factors,cc,prebatch,factor_url,holdfield=None,isdf=False,drop_func=None,future_factors={},stop_func=None):
        #获取计算图的行情
        data={}
        for _,t in self.iter_price_bycalcors_graph(obj,start,end,factors,cc,prebatch,factor_url,future_factors):
            obj0=t["obj"]
            i={}
            if holdfield:
                for k in holdfield:
                    i[k]=t.get(k,None)
            else:
                i=t
            if drop_func:
                r=drop_func(t)
                if r:
                    continue
            if obj0 not in data.keys():
                data[obj0]=[]
            data[obj0].append(i)
            if stop_func:
                r=stop_func(t)
                if r:
                    break
        data0 = {}
        if isdf:
            for obj0,v in data.items():
                vv=pd.DataFrame(v)
                data0[obj0]=vv
        else:
            data0=data
        return data0
    def iter_price(self,obj,start=None,end=None):
        """
        Args:
            obj: 支持obj,objs:有 _all目录的会更快些
            start: 开始日期，
            end: 结束日期

        Returns:正常一笔行情；考虑是否需要按因子一批行情给出一批，然后各自处理？
        """
        if isinstance(obj,str):
            start0 = self.get_startdate(obj)
            if start0 is not None:
                if str2datetime(start) < start0:
                    start = start0
        url = self.url
        if isinstance(obj,str):
            if obj != "all":
                url0 = os.path.join(url, obj)
                if os.path.exists(url0):
                    if self.freq == "%Y%m%d":
                        for d in get_calender(start,end,self.region):
                            url1=os.path.join(url0,obj+"_"+d.strftime(self.freq)+".pkl")
                            datset = self.__iml_load(url1,iszip=self.iszip)
                            if datset:
                                for i in datset:
                                    yield self.datatype,i
                            else:
                                print("sys","不存在",obj,d,url1)
                    else:
                        l = os.listdir(url0)
                        l.sort()
                        if start:
                            start = str2datetime(start)
                            if end:
                                end = str2datetime(end)
                                isstart = False
                                startstr = start.strftime(self.freq)
                                isend = False
                                for k in l:
                                    kdstr= k.replace(".pkl","").split("_")[-1]
                                    # if startstr in k:
                                    if kdstr>=startstr:
                                        isstart = True
                                    if isstart:
                                        url0=os.path.join(url, obj, k)
                                        datset=self.__iml_load(url0,iszip=self.iszip)
                                        for k,bars in datset.items():
                                            for i in bars:
                                                if i["timekey"] >= start:
                                                    if i["timekey"] <= end:
                                                        yield self.datatype,i
                                                    else:
                                                        isend = True
                                                        break
                                        if isend:
                                            break
                            else:
                                isstart = False
                                startstr = start.strftime(self.freq)
                                for k in l:
                                    kdstr = k.replace(".pkl","").split("_")[-1]
                                    # if startstr in k:
                                    if kdstr >= startstr:
                                        isstart = True
                                    if isstart:
                                        url0 = os.path.join(url, obj, k)
                                        datset = self.__iml_load(url0,iszip=self.iszip)
                                        for k, bars in datset.items():
                                            for i in bars:
                                                if i["timekey"] >= start:
                                                    yield self.datatype,i
                        else:
                            for k in l:
                                url0 = os.path.join(url, obj, k)
                                datset = self.__iml_load(url0,iszip=self.iszip)
                                for k, bars in datset.items():
                                    for i in bars:
                                        yield self.datatype,i
                else:
                    print("sys","不存在",url0)
            else:
                url0 = os.path.join(url, "_all")
                if os.path.exists(url0):
                    l = os.listdir(url0)
                    l.sort()
                    if start:
                        d0 = str2datetime(start)
                        start0 = "all_" + d0.strftime(self.freq) + ".pkl"
                    else:
                        start0 = None
                    if end:
                        d1 = str2datetime(end)
                        end0 = "all_" + d1.strftime(self.freq) + ".pkl"
                    else:
                        end0 = None
                    if start0:
                        if end0:
                            for i in l:
                                if start0 <= i <= end0:
                                    url1 = os.path.join(url0, i)
                                    dset = self.__iml_load(url1,iszip=self.iszip)
                                    if i == start0:
                                        for d, v in dset.items():
                                            if d >= d0:
                                                for i in v:
                                                    yield self.datatype,i
                                    elif i == end0:
                                        for d, v in dset.items():
                                            if d <= d1:
                                                for i in v:
                                                    yield self.datatype,i
                                    else:
                                        for d, v in dset.items():
                                            for i in v:
                                                yield self.datatype,i
                        else:
                            for i in l:
                                if i >= start0:
                                    url1 = os.path.join(url0, i)
                                    dset = self.__iml_load(url1,iszip=self.iszip)
                                    if i == start0:
                                        for d, v in dset.items():
                                            if d >= d0:
                                                for i in v:
                                                    yield self.datatype,i
                                    else:
                                        for d, v in dset.items():
                                            for i in v:
                                                yield self.datatype,i
                    else:
                        if end0:
                            for i in l:
                                if i <= end0:
                                    url1 = os.path.join(url0, i)
                                    dset = self.__iml_load(url1,iszip=self.iszip)
                                    if i == end0:
                                        for d, v in dset.items():
                                            if d <= d1:
                                                for i in v:
                                                    yield self.datatype,i
                                    else:
                                        for d, v in dset.items():
                                            for i in v:
                                                yield self.datatype,i
                        else:
                            for i in l:
                                url1 = os.path.join(url0, i)
                                dset = self.__iml_load(url1,iszip=self.iszip)
                                for d, v in dset.items():
                                    for i in v:
                                        yield self.datatype,i
                else:  # 临时拼接
                    objs = os.listdir(url)
                    for d in get_calender(start=start, end=end, region=self.region):
                        data=[]
                        for obj in objs:
                            data0=self.get_data_bydate(obj,d)
                            if data0:
                                data=data+data0
                    if data:
                        data=pd.DataFrame(data)
                        data.sort_values(by="timekey",inplace=True)
                        data=data.to_dict()
                        for i in data.values():
                            # if i["obj"] in obj:
                            yield self.datatype,i
        else:
            url0 = os.path.join(url, "_all")
            isall=len(obj)>=100
            if isall and os.path.exists(url0):
                l = os.listdir(url0)
                l.sort()
                if start:
                    d0 = str2datetime(start)
                    start0 = "all_" + d0.strftime(self.freq) + ".pkl"
                else:
                    start0 = None
                if end:
                    d1 = str2datetime(end)
                    end0 = "all_" + d1.strftime(self.freq) + ".pkl"
                else:
                    end0 = None
                if start0:
                    if end0:
                        for i in l:
                            if i >= start0 and i <= end0:
                                url1 = os.path.join(url0, i)
                                dset = self.__iml_load(url1,iszip=self.iszip)
                                if i == start0:
                                    for d, v in dset.items():
                                        if d >= d0:
                                            for i in v:
                                                if i["obj"] in obj:
                                                    yield self.datatype,i
                                elif i == end0:
                                    for d, v in dset.items():
                                        if d <= d1:
                                            for i in v:
                                                if i["obj"] in obj:
                                                    yield self.datatype,i
                                else:
                                    for d, v in dset.items():
                                        for i in v:
                                            if i["obj"] in obj:
                                                yield self.datatype,i
                    else:
                        for i in l:
                            if i >= start0:
                                url1 = os.path.join(url0, i)
                                dset = self.__iml_load(url1,iszip=self.iszip)
                                if i == start0:
                                    for d, v in dset.items():
                                        if d >= d0:
                                            for i in v:
                                                if i["obj"] in obj:
                                                    yield self.datatype,i
                                else:
                                    for d, v in dset.items():
                                        for i in v:
                                            if i["obj"] in obj:
                                                yield self.datatype,i
                else:
                    if end0:
                        for i in l:
                            if i <= end0:
                                url1 = os.path.join(url0, i)
                                dset = self.__iml_load(url1,iszip=self.iszip)
                                if i == end0:
                                    for d, v in dset.items():
                                        if d <= d1:
                                            for i in v:
                                                if i["obj"] in obj:
                                                    yield self.datatype,i
                                else:
                                    for d, v in dset.items():
                                        for i in v:
                                            if i["obj"] in obj:
                                                yield self.datatype,i
                    else:
                        for i in l:
                            url1 = os.path.join(url0, i)
                            dset = self.__iml_load(url1,iszip=self.iszip)
                            for d, v in dset.items():
                                for i in v:
                                    if i["obj"] in obj:
                                        yield self.datatype,i
            else:  # 临时拼接
                for d in get_calender(start=start, end=end, region=self.region):
                    data = []
                    for obj0 in obj:
                        data0 = self.get_data_bydate(obj0, d)
                        if data0:
                            data.extend(data0)
                    if data:
                        data = pd.DataFrame(data)
                        data.sort_values(by="timekey", inplace=True)
                        data = data.to_dict("index")
                        for i in data.values():
                            yield self.datatype,i
    def iter_data_addk1d(self,obj,start=None,end=None):#策略框架使用，k1d必须，结算需要，日前日后触发需要
        url = self.url
        if self.datatype=="k1d":
            for i in self.iter_price(obj,start,end):
                yield i
        else:
            k1d_dataset=dataset(product=self.product,datatype="k1d",baseurl=self.baseurl,region=self.region,fqtype=self.fqtype,source=self.source)
            start1 = start
            if isinstance(obj,str):
                start0=k1d_dataset.get_startdate(obj)
                if start0 is not None:
                    if str2datetime(start) < start0:
                        start1 = start0
                # end0=k1d_dataset.get_maxdate(obj)
                # if end0 is not None:
                #     if end0<end:
                #         end=end0
            if isinstance(obj,str):
                if obj != "all":
                    url0 = os.path.join(url, obj)
                    if os.path.exists(url0):
                        if self.freq == "%Y%m%d":
                            for d in get_calender(start1, end, self.region):
                                url1=os.path.join(url0,obj+"_"+d.strftime(self.freq)+".pkl")
                                datset = self.__iml_load(url1,iszip=self.iszip)
                                if datset:
                                    for i in datset:
                                        yield self.datatype, i
                                else:
                                    print("sys", "不存在", obj, d, url1)
                                    break
                                k1d0=k1d_dataset.get_data_bydate(obj,d)
                                for i in k1d0:
                                    yield "k1d",i
                        else:
                            preurl1=None
                            for d in get_calender(start1, end, self.region):
                                url1=os.path.join(url0,obj+"_"+d.strftime(self.freq)+".pkl")
                                if url1!=preurl1:
                                    datset=self.__iml_load(url1,iszip=self.iszip)
                                    preurl1=url1
                                    if datset   is None:
                                        datset={}
                                if d in datset.keys():
                                    for i in datset[d]:
                                        yield self.datatype, i
                                else:
                                    print("sys", "不存在", obj, d, url1)
                                    break
                                k1d0=k1d_dataset.get_data_bydate(obj,d)
                                for i in k1d0:
                                    yield "k1d", i
                    else:
                        print("sys","不存在",obj,url0)
                else:
                    url0 = os.path.join(url, "_all")
                    if os.path.exists(url0):
                        l = os.listdir(url0)
                        l.sort()
                        if start1:
                            d0 = str2datetime(start1)
                            start0 = "all_" + d0.strftime(self.freq) + ".pkl"
                        else:
                            start0 = None
                        if end:
                            d1 = str2datetime(end)
                            end0 = "all_" + d1.strftime(self.freq) + ".pkl"
                        else:
                            end0 = None
                        if start0:
                            if end0:
                                for i in l:
                                    if i <= start0 and i <= end0:
                                        url1 = os.path.join(url0, i)
                                        dset = self.__iml_load(url1,iszip=self.iszip)
                                        if i == start0:
                                            for d, v in dset.items():
                                                if d >= d0:
                                                    for i in v:
                                                        if i["obj"] in obj:
                                                            yield self.datatype,i
                                        elif i == end0:
                                            for d, v in dset.items():
                                                if d <= d1:
                                                    for i in v:
                                                        if i["obj"] in obj:
                                                            yield self.datatype,i
                                        else:
                                            for d, v in dset.items():
                                                for i in v:
                                                    if i["obj"] in obj:
                                                        yield self.datatype,i
                            else:
                                for i in l:
                                    if i <= start0:
                                        url1 = os.path.join(url0, i)
                                        dset = self.__iml_load(url1,iszip=self.iszip)
                                        if i == start0:
                                            for d, v in dset.items():
                                                if d >= d0:
                                                    for i in v:
                                                        if i["obj"] in obj:
                                                            yield self.datatype,i
                                        else:
                                            for d, v in dset.items():
                                                for i in v:
                                                    if i["obj"] in obj:
                                                        yield self.datatype,i
                        else:
                            if end0:
                                for i in l:
                                    if i <= end0:
                                        url1 = os.path.join(url0, i)
                                        dset = self.__iml_load(url1,iszip=self.iszip)
                                        if i == end0:
                                            for d, v in dset.items():
                                                if d <= d1:
                                                    for i in v:
                                                        if i["obj"] in obj:
                                                            yield self.datatype,i
                                        else:
                                            for d, v in dset.items():
                                                for i in v:
                                                    if i["obj"] in obj:
                                                        yield self.datatype,i
                            else:
                                for i in l:
                                    url1 = os.path.join(url0, i)
                                    dset = self.__iml_load(url1,iszip=self.iszip)
                                    for d, v in dset.items():
                                        for i in v:
                                            if i["obj"] in obj:
                                                yield self.datatype,i
                    else:  # 临时拼接
                        objs = os.listdir(url)
                        for d in get_calender(start=start1, end=end, region=self.region):
                            data=[]
                            for obj in objs:
                                data0=self.get_data_bydate(obj,d)
                                if data0:
                                    data=data.extend(data0)
                        if data:
                            data=pd.DataFrame(data)
                            data.sort_values(by="timekey",inplace=True)
                            data=data.to_dict()
                            for i in data.values():
                                if i["obj"] in obj:
                                    yield self.datatype,i
            else:
                url0 = os.path.join(url, "_all")
                if os.path.exists(url0):
                    l = os.listdir(url0)
                    l.sort()
                    if start1:
                        d0 = str2datetime(start1)
                        start0 = "all_" + d0.strftime(self.freq) + ".pkl"
                    else:
                        start0 = None
                    if end:
                        d1 = str2datetime(end)
                        end0 = "all_" + d1.strftime(self.freq) + ".pkl"
                    else:
                        end0 = None
                    if start0:
                        if end0:
                            for i in l:
                                if i <= start0 and i <= end0:
                                    url1 = os.path.join(url0, i)
                                    dset = self.__iml_load(url1,iszip=self.iszip)
                                    if i == start0:
                                        for d, v in dset.items():
                                            if d >= d0:
                                                for i in v:
                                                    yield self.datatype,i
                                    elif i == end0:
                                        for d, v in dset.items():
                                            if d <= d1:
                                                for i in v:
                                                    yield self.datatype,i
                                    else:
                                        for d, v in dset.items():
                                            for i in v:
                                                yield self.datatype,i
                        else:
                            for i in l:
                                if i <= start0:
                                    url1 = os.path.join(url0, i)
                                    dset = self.__iml_load(url1,iszip=self.iszip)
                                    if i == start0:
                                        for d, v in dset.items():
                                            if d >= d0:
                                                for i in v:
                                                    yield self.datatype,i
                                    else:
                                        for d, v in dset.items():
                                            for i in v:
                                                yield self.datatype,i
                    else:
                        if end0:
                            for i in l:
                                if i <= end0:
                                    url1 = os.path.join(url0, i)
                                    dset = self.__iml_load(url1,iszip=self.iszip)
                                    if i == end0:
                                        for d, v in dset.items():
                                            if d <= d1:
                                                for i in v:
                                                    yield self.datatype,i
                                    else:
                                        for d, v in dset.items():
                                            for i in v:
                                                yield self.datatype,i
                        else:
                            for i in l:
                                url1 = os.path.join(url0, i)
                                dset = self.__iml_load(url1,iszip=self.iszip)
                                for d, v in dset.items():
                                    for i in v:
                                        yield self.datatype,i
                else:  # 临时拼接
                    objs = os.listdir(url)
                    for d in get_calender(start=start1, end=end, region=self.region):
                        data = []
                        for obj in objs:
                            data0 = self.get_data_bydate(obj, d)
                            if data0:
                                data = data.extend(data0)
                    if data:
                        data = pd.DataFrame(data)
                        data.sort_values(by="timekey", inplace=True)
                        data = data.to_dict()
                        for i in data.values():
                            yield self.datatype,i
    #历史数据导入导出工具
    def export_data1(self,obj,freq1,url0,url1):
        try:
            print("deal",obj)
            l = os.listdir(os.path.join(url0, obj))
            l.sort()
            prek = None
            data = {}
            for i in l:
                d = i.replace(".pkl","").split("_")[-1]
                d = str2datetime(d)
                key = d.strftime(freq1)
                if prek   is None:
                    prek = key
                url = os.path.join(url0, obj, i)
                data0 = self.__iml_load(url,iszip=self.iszip)
                if prek == key:
                    data[d] = data0
                else:
                    url11 = os.path.join(url1, obj, obj + "_" + prek + ".pkl")
                    self.__iml_save(data, url11,iszip=self.iszip)
                    prek = key
                    data = {}
                    data[d] = data0
            url11 = os.path.join(url1, obj, obj + "_" + prek + ".pkl")
            self.__iml_save(data, url11,iszip=self.iszip)
        except Exception as e:
            print(url)
            print(e)
    def export_data2(self,obj,url0,url1):
        print("deal",obj)
        try:
            l = os.listdir(os.path.join(url0, obj))
            l.sort()
            for i in l:
                url = os.path.join(url0, obj, i)
                data = self.__iml_load(url,iszip=self.iszip)
                self.self.__iml_save(obj, data, url=url1)
        except Exception as e:
            print(url)
            print(e)
    def export_new_data(self,freq0,freq1,url0,url1,obj0=None,pool_n=1):
        objs=os.listdir(url0)
        isrun=True
        if obj0:
            isrun=False
        if pool_n>1:
            pool = multiprocessing.Pool(pool_n)
        if len(freq0)>len(freq1):
            for obj in objs:
                if not isrun:
                    if obj0==obj:
                        isrun=True
                    else:
                        print("跳过",obj)
                        continue
                if pool_n > 1:
                    pool.apply_async(self.export_data1,(obj,freq1,url0,url1))
                else:
                    self.export_data1(obj,freq1,url0,url1)

        elif len(freq0)<len(freq1):
            for obj in objs:
                if not isrun:
                    if obj0==obj:
                        isrun=True
                    else:
                        print("跳过",obj)
                        continue
                if pool_n > 1:
                    pool.apply_async(self.export_data2,(obj, url0, url1))
                else:
                    self.export_data2(obj, url0, url1)

        if pool_n > 1:
            pool.close()  # 关闭进程池，不再接受新的进程
            pool.join()  # 主进程阻塞等待子进程的退出





if __name__=="__main__":
    multiprocessing.freeze_support()
    # url0=r"I:\price\stock\k5m\hfq"
    # url1 = r"I:\price\stock\k5m2\hfq"
    # dataset0=dataset(product="stock",objs=None,datatype="k5m",baseurl=r"F:",region="C",url=None,fqtype="hfq")
    # dataset0.export_new_data(freq0="%Y%m%d",freq1="%Y%m",url0=url0,url1=url1,obj0="600027",pool_n=20)
    data={"tick":[],"kline":[]}
    for i in [("stock","000001",{"k1d":"hfq","k5m":"hfq","k1m":"bfq","tick":"bfq"}),
              ("fund","510300",{"k1d":"bfq","k1m":"bfq"}),
              ("index","000001",{"k1d":"bfq"}),
              ("future","TA",{"k1d":"hfq"}),
              ("future", "TA209", {"k1m":"hfq","tick":"hfq"})]:
        for k,v in i[2].items():
            dataset0 = dataset(product=i[0], objs=None, datatype=k, baseurl=r"G:\price", region="C", url=None,
                               fqtype=v)
            dd=dataset0.get_lastdata(i[1])
            dd["datatype"]=k
            dd["fqtype"]=v
            if k =="tick":
                data[k].append(dd)
            else:
                data["kline"].append(dd)
            print(i[0],i[1],k,dd)
    # import pandas as pd
    # ksd = pd.DataFrame(data["kline"])
    # ksd1 = pd.DataFrame(data["tick"])
    from jili.tool.tofile import df_to_excel
    df_to_excel(data,url=r"G:/price/kline_tick_data.xlsx",name=None,fontsize=10)
